package test.spider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.Response;

public class App2 {
	
	static ExecutorService threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
	
	static CopyOnWriteArraySet<String> urlhashs = new CopyOnWriteArraySet<>();
	
	static AtomicInteger totalCount = new AtomicInteger(0);
	static AtomicInteger failCount = new AtomicInteger(0);
	
	static LinkedBlockingQueue<Object[]> queue = new LinkedBlockingQueue<>();
	
	static int level = 3;
	
	public static void main(String[] args) {
		long l1 = System.currentTimeMillis();
		recur(new ArrayList<String>(), "http://licai.51credit.com");
		
		listen();
		
		Runtime.getRuntime().addShutdownHook(new Thread() {
			@Override
			public void run() {
				long l2 = System.currentTimeMillis();
				System.out.println("共耗时："+(l2-l1)+"ms");
				System.out.println("共查出"+totalCount.get()+"条记录");
				System.out.println("有"+totalCount.get()+"条记录请求失败");
			}
		});
	}
	
	static void recur(List<String> parents, String url) {
		if(parents.size()==level) return;
		OkHttpClient client = new OkHttpClient();
		try {
			Request request = new Request.Builder().url(url).build();
			String Host = getHost(url);
			Response response = client.newCall(request).execute();
			String html = response.body().string();
			Document doc = Jsoup.parse(html);
			Elements elements = doc.select("a[href]");
			for(Element element : elements) {
				String surl = element.attr("href").trim();
				//过滤非超链接
				if(surl.equals("")||surl.equals("javascript:;")||surl.equals("javascript:void(0);")) continue;
				//替换相对链接
				if(surl.startsWith("/")) surl = Host + surl;
				//其他网站跳过
				if(!surl.startsWith(Host)) continue;
				//抓过的链接跳过
				if(urlhashs.contains(surl)) continue;
				else urlhashs.add(surl);
				
				List<String> temp = new ArrayList<>(parents);
				temp.add(url);
				queue.add(new Object[]{temp, surl});
				
			}
		} catch (IOException e) {
			System.out.println(e.getMessage());
			failCount.incrementAndGet();
		}
	}
	
	static void listen() {
		while(true) {
			try {
				Thread.sleep(100);
				if(queue.isEmpty()) break;
				Object[] objs = queue.take();
				threadPool.submit(new Runnable() {
					@Override
					public void run() {
						@SuppressWarnings("unchecked")
						List<String> parents = (List<String>) objs[0];
						String url = (String) objs[1];
						if(url.matches("http://licai.51credit.com/article/detail/.+")) {
							System.out.println(parents);
							System.out.println("----->"+url);
							totalCount.incrementAndGet();
						} else {
							recur(parents, url);
						}
					}
				});
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		
		threadPool.shutdown();
	}
	
	static String getHost(String url) {
		int index = url.indexOf("/", 7);
		if(index == -1) index = url.length();
		return url.substring(0, index);
	}
	
//	public static void main(String[] args) {
//		System.out.println(getHost("http://licai.51credit.com/aa/bbb"));
//	}

}
